import os.path

from requests import get
from lxml import etree
from filer_toos import mkdir


def get_rank():
    ret = get('https://www.zongheng.com/rank.html')
    html = etree.HTML(ret.text)
    yp_rank = html.xpath('/html/body/div[2]/div[4]/div[2]/div[2]/div[1]')[0]
    rank_name = yp_rank.xpath('./div[1]')[0].text
    mkdir(f"./{rank_name}")
    a_list = yp_rank.xpath("./div/div[contains(@class,'rank_i_bname')]/a[1]")
    book_map = {}
    for a in a_list:
        book_name = a.text.replace(' ', '_')
        book_url = a.get('href').replace('/book/', '/showchapter/')
        book_map[book_name] = book_url
    return book_map, f"./{rank_name}/"


def get_chapter(url, path):
    mkdir(path)
    ret = get(url)
    html = etree.HTML(ret.text)
    chapter_list = html.xpath('/html/body/div[3]/div[2]/div[2]/div/ul/li/a')
    chapter_map = {}
    for a in chapter_list:
        book_name = a.text.replace(' ', '_')
        book_url = a.get('href')
        chapter_map[book_name] = book_url
    return chapter_map, path


# 小说内容
def get_content(url, chapter_name, path):
    ret = get(url)

    html = etree.HTML(ret.text)
    lines = html.xpath('//*[@id="readerFt"]/div/div[1]/div[5]/p')

    with open(os.path.join(path, chapter_name), 'w') as f:
        f.write("\n".join([p.text for p in lines]))


if __name__ == '__main__':
    book_data, path = get_rank()
    for name, url in book_data.items():
        print(f"正在爬取   {name}")

        chapter_data, book_path = get_chapter(url, path + name)

        # 增量过滤
        locals_chapter = set(os.listdir(book_path))
        [chapter_data.pop(key) for key in locals_chapter]

        for chapter_name, chapter_url in chapter_data.items():
            print(chapter_name)
            get_content(chapter_url, chapter_name, book_path)
